{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "db68f9ed-b3d6-4722-92da-d23681ceaab2",
   "metadata": {},
   "source": [
    "# Exercise 1\n",
    "\n",
    "## Supervised Learning- Regression Models\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "## Overview\n",
    "\n",
    "Welcome to this excercise. We are now going to use our new skills to build our supervised learning models that use a regression approach.\n",
    "\n",
    "---\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a5cbe436",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting statsmodels\n",
      "  Downloading statsmodels-0.14.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (10.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.7/10.7 MB\u001b[0m \u001b[31m55.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scipy!=1.9.2,>=1.8 in /opt/venv/lib/python3.10/site-packages (from statsmodels) (1.14.1)\n",
      "Collecting patsy>=0.5.6\n",
      "  Downloading patsy-1.0.1-py2.py3-none-any.whl (232 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m232.9/232.9 kB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: pandas!=2.1.0,>=1.4 in /opt/venv/lib/python3.10/site-packages (from statsmodels) (2.2.3)\n",
      "Requirement already satisfied: packaging>=21.3 in /opt/venv/lib/python3.10/site-packages (from statsmodels) (24.1)\n",
      "Requirement already satisfied: numpy<3,>=1.22.3 in /opt/venv/lib/python3.10/site-packages (from statsmodels) (2.1.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /opt/venv/lib/python3.10/site-packages (from pandas!=2.1.0,>=1.4->statsmodels) (2024.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/venv/lib/python3.10/site-packages (from pandas!=2.1.0,>=1.4->statsmodels) (2024.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/venv/lib/python3.10/site-packages (from pandas!=2.1.0,>=1.4->statsmodels) (2.9.0.post0)\n",
      "Requirement already satisfied: six>=1.5 in /opt/venv/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas!=2.1.0,>=1.4->statsmodels) (1.16.0)\n",
      "Installing collected packages: patsy, statsmodels\n",
      "Successfully installed patsy-1.0.1 statsmodels-0.14.5\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install statsmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cd4b8b09-f9e0-4c1f-9885-1aef53e84bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "# Manually set the path relative to the py file's location that you want to import\n",
    "func_lib_path = os.path.abspath(os.path.join(os.getcwd(), '../../'))# Add the path to sys.path\n",
    "sys.path.append(func_lib_path)\n",
    "\n",
    "# Now you can import func_lib\n",
    "import func_lib\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import statsmodels.api as sm\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c82294fa-8583-43e2-9f50-be891efdef8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/CD13639-Project/func_lib.py:12: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  sp500_tickers = pd.read_html(response.text)[0]['Symbol'].tolist()\n",
      "[**********************91%*******************    ]  455 of 501 completed"
     ]
    }
   ],
   "source": [
    "# Retrieve historical price data.\n",
    "# - Use the 'createHistPrices()' function from 'func_lib' to get historical price data.\n",
    "# - Assign the result to the variable 'historical_prices'.\n",
    "\n",
    "historical_prices = func_lib.create_hist_prices()\n",
    "\n",
    "# Define the list of momentum periods.\n",
    "# - Create a list containing different time periods for calculating momentum indicators.\n",
    "# - Assign this list to the variable 'list_of_momentums'.\n",
    "\n",
    "list_of_momentums = [1, 5, 15, 20]\n",
    "\n",
    "# Compute returns based on historical prices and momentum periods.\n",
    "# - Use the 'computingReturns()' function from 'func_lib' with 'historical_prices' and 'list_of_momentums' as inputs.\n",
    "# - Assign the result to the variable 'total_returns'.\n",
    "\n",
    "total_returns = func_lib.compute_returns(historical_prices, list_of_momentums)\n",
    "\n",
    "# Remove any rows with missing values from the returns data.\n",
    "# - Use the 'dropna()' method to remove rows with NaN values from 'total_returns'.\n",
    "# - Apply the change in place to modify the DataFrame directly.\n",
    "\n",
    "total_returns.dropna(inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5967d6de-5270-4a3e-969e-7a3fbcce2d62",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Determine the split index for 70% of the dates.\n",
    "# - Extract the unique dates from the index of 'total_returns'.\n",
    "# - Calculate the split date by indexing into the unique dates to get the date corresponding to 70% of the total length of unique dates.\n",
    "# - Assign this date to the variable 'split_date' and display it.\n",
    "\n",
    "unique_dates = total_returns.index.get_level_values('Date').unique()\n",
    "split_date = unique_dates[int(0.7 * len(unique_dates))]\n",
    "print(split_date)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c02c2d11-1505-4bb5-8bd8-3b3e6698f1f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the training set: all data before the split date.\n",
    "# - Use the 'loc[]' indexer to select rows from 'total_returns' where the date in the index is earlier than 'split_date'.\n",
    "# - Assign this subset of data to the variable 'train_data'.\n",
    "\n",
    "train_data = total_returns.loc[total_returns.index.get_level_values('Date') < split_date]\n",
    "\n",
    "# Create the testing set: all data from the split date onwards.\n",
    "# - Use the 'loc[]' indexer to select rows from 'total_returns' where the date in the index is equal to or later than 'split_date'.\n",
    "# - Assign this subset of data to the variable 'test_data'.\n",
    "\n",
    "test_data = total_returns.loc[total_returns.index.get_level_values('Date') >= split_date]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f42ac29-0062-4894-a158-2173eb917714",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract the target variable for testing.\n",
    "# - Select the 'F_1_d_returns' column from the 'test_data' DataFrame.\n",
    "# - Assign this column to the variable 'total_returns'.\n",
    "\n",
    "total_returns = test_data['F_1_d_returns']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee5782b5-3c6e-41da-a17d-b52f01a4359d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the feature and target columns.\n",
    "# - List the names of the feature columns in 'features'.\n",
    "# - List the name of the target column in 'target'.\n",
    "\n",
    "features = ['1_d_returns', '5_d_returns', '15_d_returns', '20_d_returns']\n",
    "target   = ['F_1_d_returns']\n",
    "\n",
    "# Split the data into training and testing sets.\n",
    "# - Extract the feature columns from 'train_data' and 'test_data' to create 'X_train' and 'X_test'.\n",
    "# - Extract the target column from 'train_data' and 'test_data' to create 'y_train' and 'y_test'.\n",
    "\n",
    "X_train = train_data[features]\n",
    "X_test  = test_data[features]\n",
    "y_train = train_data[target]\n",
    "y_test  = test_data[target]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed9e2980-0253-4b63-92f3-751918ccd4e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standardize the features.\n",
    "# - Create an instance of 'StandardScaler' for scaling the features.\n",
    "# - Fit the scaler to the training data and transform both the training and testing feature sets.\n",
    "# - Convert the scaled arrays back to DataFrames with the same index and column names as the original feature sets.\n",
    "\n",
    "scaler = StandardScaler()\n",
    "\n",
    "# Scale the training features\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "# Scale the testing features\n",
    "X_test_scaled  = scaler.transform(X_test)\n",
    "\n",
    "# Convert the scaled arrays back to DataFrames\n",
    "X_train_scaled = pd.DataFrame(X_train_scaled, index=X_train.index, columns=X_train.columns)\n",
    "X_test_scaled  = pd.DataFrame(X_test_scaled, index=X_test.index, columns=X_test.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1d6724a-1094-4ae9-8e9b-b032676cbae9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a constant to the model (intercept).\n",
    "# - Use 'sm.add_constant()' to add a column of ones to the scaled feature sets for both training and testing data.\n",
    "# - This column represents the intercept term in the regression model.\n",
    "\n",
    "X_train_scaled_const_add = sm.add_constant(X_train_scaled)\n",
    "X_test_scaled_const_add  = sm.add_constant(X_test_scaled)\n",
    "\n",
    "# Fit the model.\n",
    "# - Create an Ordinary Least Squares (OLS) regression model using 'sm.OLS()' with the scaled training features and the target variable.\n",
    "# - Fit the model to the training data.\n",
    "\n",
    "model = sm.OLS(y_train, X_train_scaled_const_add).fit()\n",
    "\n",
    "# Print the model summary.\n",
    "# - Use 'model.summary()' to output the regression results, including coefficients, R-squared, and other statistical metrics.\n",
    "\n",
    "print(model.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f39393e-1f82-41df-b657-c5f262bcf0cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the model name.\n",
    "# - Assign the name 'Reg' to the variable 'model_name' to label the regression model.\n",
    "\n",
    "model_name = 'Reg'\n",
    "\n",
    "# Make predictions using the fitted model.\n",
    "# - Use 'model.predict()' to generate predictions based on the scaled testing features with the added constant.\n",
    "\n",
    "y_pred = model.predict(X_test_scaled_const_add)\n",
    "\n",
    "# Create DataFrames for the actual and predicted values.\n",
    "# - Convert the true target values 'y_test' and predicted values 'y_pred' to DataFrames.\n",
    "# - Rename the column in 'y_pred_df' to match the model name.\n",
    "\n",
    "y_test_df = pd.DataFrame(y_test)\n",
    "y_pred_df = pd.DataFrame(y_pred)\n",
    "y_pred_df.rename(columns={0: model_name}, inplace=True)\n",
    "\n",
    "# Combine the actual and predicted values into one DataFrame.\n",
    "# - Use 'pd.merge()' to merge 'y_pred_df' with 'total_returns' on their index.\n",
    "# - This combines the predicted values with the actual returns data.\n",
    "\n",
    "y_test_and_pred = pd.merge(y_pred_df, total_returns, left_index=True, right_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82f6ecce-d8d6-41da-acb5-afd6dc10bbc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a trading strategy based on logistic regression predictions.\n",
    "# - Create a function 'trading_strategy' that takes the predicted value 'y_pred' as input.\n",
    "# - If the predicted value is greater than 0, return 1 (indicating a \"Go long\" signal).\n",
    "# - Otherwise, return 0 (indicating no action or \"Do nothing\").\n",
    "\n",
    "def trading_strategy(y_pred):\n",
    "    if y_pred > 0:\n",
    "        return 1  # Go long\n",
    "    else:\n",
    "        return 0  # Do nothing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83243879-11f9-4e4c-abce-24f662fc0c37",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute performance metrics based on the predictions and actual returns.\n",
    "# - Use the 'func_lib.compute_BM_Perf()' function to calculate cumulative and calendar returns.\n",
    "# - Pass the DataFrame 'y_test_and_pred' containing the actual and predicted values as an argument.\n",
    "\n",
    "cum_returns, calendar_returns = func_lib.compute_BM_Perf(y_test_and_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a446adf2-bbf2-486c-9c63-0a26239aa26a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute strategy performance metrics.\n",
    "# - Use the 'func_lib.compute_strat_perf()' function to calculate the performance of the trading strategy.\n",
    "# - Pass the following arguments:\n",
    "#   - 'y_test_and_pred': The DataFrame containing actual and predicted values.\n",
    "#   - 'cum_returns': Cumulative returns calculated previously.\n",
    "#   - 'calendar_returns': Calendar returns calculated previously.\n",
    "#   - 'trading_strategy': The function defining the trading strategy based on predictions.\n",
    "#   - 'model_name': The name of the model to include in the performance evaluation.\n",
    "\n",
    "cum_returns, calendar_returns = func_lib.compute_strat_perf(\n",
    "    y_test_and_pred, \n",
    "    cum_returns, \n",
    "    calendar_returns, \n",
    "    trading_strategy, \n",
    "    model_name\n",
    ")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
